

<!DOCTYPE html>
<html lang="zh-CN" data-default-color-scheme=&#34;light&#34;>



<head>
  <meta charset="UTF-8">
  <link rel="apple-touch-icon" sizes="76x76" href="https://cdn.jsdelivr.net/gh/kzwrime/kzwrime.github.io/img/favicon.png">
  <link rel="icon" type="image/png" href="https://tuchuangrrr.oss-cn-shenzhen.aliyuncs.com/img/kunnan.png">
  <meta name="viewport"
        content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, shrink-to-fit=no">
  <meta http-equiv="x-ua-compatible" content="ie=edge">
  
  <meta name="theme-color" content="#2f4154">
  <meta name="description" content="">
  <meta name="author" content="霜华">
  <meta name="keywords" content="">
  <title>Automatic Parallelization Note 01 - kunzh</title>

  <link  rel="stylesheet" href="https://cdn.staticfile.org/twitter-bootstrap/4.4.1/css/bootstrap.min.css" />


  <link  rel="stylesheet" href="https://cdn.staticfile.org/github-markdown-css/4.0.0/github-markdown.min.css" />
  <link  rel="stylesheet" href="/lib/hint/hint.min.css" />

  
    
    
      
      <link  rel="stylesheet" href="https://cdn.staticfile.org/highlight.js/10.0.0/styles/atelier-lakeside-dark.min.css" />
    
  

  


<!-- 主题依赖的图标库，不要自行修改 -->

<link rel="stylesheet" href="//at.alicdn.com/t/font_1749284_pf9vaxs7x7b.css">



<link rel="stylesheet" href="//at.alicdn.com/t/font_1736178_kmeydafke9r.css">


<link  rel="stylesheet" href="/css/main.css" />

<!-- 自定义样式保持在最底部 -->


  <script  src="/js/utils.js" ></script>
  <script  src="/js/color-schema.js" ></script>
</head>


<body>
  <header style="height: 60vh;">
    <nav id="navbar" class="navbar fixed-top  navbar-expand-lg navbar-dark scrolling-navbar">
  <div class="container">
    <a class="navbar-brand"
       href="/">&nbsp;<strong>kunzh</strong>&nbsp;</a>

    <button id="navbar-toggler-btn" class="navbar-toggler" type="button" data-toggle="collapse"
            data-target="#navbarSupportedContent"
            aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
      <div class="animated-icon"><span></span><span></span><span></span></div>
    </button>

    <!-- Collapsible content -->
    <div class="collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav ml-auto text-center">
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/">
                <i class="iconfont icon-home-fill"></i>
                首页
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/archives/">
                <i class="iconfont icon-archive-fill"></i>
                归档
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/categories/">
                <i class="iconfont icon-category-fill"></i>
                分类
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/tags/">
                <i class="iconfont icon-tags-fill"></i>
                标签
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/about/">
                <i class="iconfont icon-user-fill"></i>
                关于
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/links/">
                <i class="iconfont icon-link-fill"></i>
                友链
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/shuoshuo2/">
                <i class="iconfont icon-speakernotes"></i>
                留言板
              </a>
            </li>
          
        
        
          <li class="nav-item" id="search-btn">
            <a class="nav-link" data-toggle="modal" data-target="#modalSearch">&nbsp;<i
                class="iconfont icon-search"></i>&nbsp;</a>
          </li>
        
        
          <li class="nav-item" id="color-toggle-btn">
            <a class="nav-link" href="javascript:">&nbsp;<i
                class="iconfont icon-dark" id="color-toggle-icon"></i>&nbsp;</a>
          </li>
        
      </ul>
    </div>
  </div>
</nav>

    <div class="banner intro-2" id="background" parallax=true
         style="background: url('https://rainrime.top/img/SloveniaAlps.jpg') no-repeat center center;
           background-size: cover;">
      <div class="full-bg-img">
        <div class="mask flex-center" style="background-color: rgba(0, 0, 0, 0.3)">
          <div class="container page-header text-center fade-in-up">
            <span class="h2" id="subtitle">
              
            </span>

            
              <div class="mt-3">
  
  
    <span class="post-meta">
      <i class="iconfont icon-date-fill" aria-hidden="true"></i>
      <time datetime="2020-11-03 14:50" pubdate>
        2020年11月3日 下午
      </time>
    </span>
  
</div>

<div class="mt-1">
  
    
    <span class="post-meta mr-2">
      <i class="iconfont icon-chart"></i>
      3.7k 字
    </span>
  

  
    
    <span class="post-meta mr-2">
      <i class="iconfont icon-clock-fill"></i>
      
      
      42
       分钟
    </span>
  

  
  
    
      <!-- LeanCloud 统计文章PV -->
      <span id="leancloud-post-views-container" class="post-meta" style="display: none">
        <i class="iconfont icon-eye" aria-hidden="true"></i>
        <span id="leancloud-post-views"></span> 次
      </span>
    
  
</div>

            
          </div>

          
        </div>
      </div>
    </div>
  </header>

  <main>
    
      

<div class="container-fluid">
  <div class="row">
    <div class="d-none d-lg-block col-lg-2"></div>
    <div class="col-lg-8 nopadding-md">
      <div class="container nopadding-md" id="board-ctn">
        <div class="py-5" id="board">
          <article class="post-content mx-auto" id="post">
            <!-- SEO header -->
            <h1 style="display: none">Automatic Parallelization Note 01</h1>
            
            <div class="markdown-body" id="post-body">
              <h1 id="automatic-parallelization-note-01"><a class="markdownIt-Anchor" href="#automatic-parallelization-note-01"></a> Automatic Parallelization Note 01</h1>
<p>​		并行化编译技术，特别是<strong>循环体的并行化编译</strong>技术，越来越受到重视。<strong>TLS</strong>(Thread level speculation)编译，即猜测并行化编译，可将原来顺序运行的程序并行化。</p>
<h4 id="主要问题"><a class="markdownIt-Anchor" href="#主要问题"></a> 主要问题</h4>
<ul>
<li>并行化算法:如何进行任务的划分，如何分配才是最优</li>
<li>资源的利用:并行程序开发者需要考虑资源如何才能有效利用，需要考虑负载平衡、软件硬件并行开销、缓存开销以及资源竞争等问题。</li>
</ul>
<h4 id="传统并行优化要点及优缺点"><a class="markdownIt-Anchor" href="#传统并行优化要点及优缺点"></a> 传统并行优化要点及优缺点</h4>
<ul>
<li>已有热点代码的并行性能提升
<ul>
<li>手动分析源码/反汇编，分析和确认热点，再对热点代码进行重点并行优化
<ul>
<li>缺点：繁琐，源代码混乱时困难</li>
</ul>
</li>
<li>利用相关 Profiling 工具分析出代码热点，再次优化代码
<ul>
<li>缺点：任需要预运行，当只能运行一次时无效</li>
</ul>
</li>
<li>优点：精确地确认热点，可以使用多种传统优化技术，调用高性能计算库，乃至手写汇编获得最高性能。</li>
</ul>
</li>
<li>直接并行编程
<ul>
<li>依赖于大量的同步操作，开销高可拓展性差</li>
<li>软硬件架构复杂化的背景下，对程序员挖掘线程级并行性(TLP)和数据级并行性(DLP)提出了更高的挑战</li>
</ul>
</li>
<li>显式并行编程（如 pthread )
<ul>
<li>通过提供细粒度的接口，使程序员有较好的调度和优化空间</li>
<li>较高的复杂度和较低的效率，易出错</li>
</ul>
</li>
<li>隐式并行编程（如 OpenMP ）
<ul>
<li>接口抽象程度更高，标注简单</li>
<li>调度和优化空间小，有部分限制如循环代码格式、不能随意分支等</li>
</ul>
</li>
</ul>
<h3 id="循环及数据相关"><a class="markdownIt-Anchor" href="#循环及数据相关"></a> 循环及数据相关</h3>
<p>Polychronopou-los和Kuck提出的分类法是最早的分类法之一。他们将循环分为三种不同的类型，即 DOALL，FORALL和 DOACROSS。</p>
<ul>
<li>
<p>**DOALL循环。**循环迭代之间不存在相关关系，意味着所有迭代可以并行处理而无需进一步检查。图1.2 (a）表示这类循环。目前大多数编译器可以自动并行化这种循环。</p>
</li>
<li>
<p>**FORALL循环。**循环迭代中可能呈现为真(RAW)数据相关关系，即由一个迭代所生成的值在随后的迭代中被使用。具体例子如图1.2 (b）所示。当且仅当产生该值的所有语句(图中的S）在使用该值的其他语句(图中的Sz）执行之前完成时，可以同时执行 forall 循环的所有迭代;如果无法保证此顺序，则需要同步机制。</p>
</li>
<li>
<p><strong>DOACROSS循环</strong>。循环具有交叉迭代（也称为WAR)反相关关系。Krotha-palli和 Sadayappan [1990]将DOACROSS循环进一步分为以下三类。</p>
</li>
<li>
<p><u>常规DOACROSS 循环</u>。循环迭代之间的反相关关系是由特定值x决定。图1.2 (c）表示了一个例子。通过确保包含相关性的迭代执行遵循顺序语义，可以并行化所有满足a &gt;1的常规 DOACROSS 循环。如果r的值在编译时已知，则编译器能够生成并行版本的循环。</p>
</li>
<li>
<p><u>DOSEQUENTIAL 或 DOSERIAL 循环</u>。这是一种特殊的常规 DOACROSS 循环。其迭代取决于前一个循环（即具有相关距离x= 1的循环)。图1.2 (d)显式了一个例子，其中的数据相关发生在循环体的最后一个语旬(本次迭代)和第一个语旬(上一次迭代)之间。这种循环在迭代之间没有并行性。</p>
<ul>
<li><u>非常规的 DOACROSS 循环</u>。循环中的反相关在编译时无法确定。图1.2(e)表示了一个例子。这些循环通常被称为“不规则循环”。</li>
</ul>
</li>
</ul>
<img src="https://tuchuangrrr.oss-cn-shenzhen.aliyuncs.com/img/image-20201102200940753.png" srcset="/img/loading.gif" alt="image-20201102200940753" style="zoom:67%;" />
<p>对于DOACROSS循环的研究已有多年历史。现有主流的自动化并行策略主要包括以下三种策略：</p>
<ul>
<li>
<p>Unnikrishnan 在文章《A practical approach to DOACROSS  parallelization》中认为单层的并行性足以开发可利用资源，只考虑了规则DOACROSS循环(即能够静态确定依赖关系的DOACROSS循环)，提出了一种 complier-and-runtime 的方法，称为<strong>依赖折叠(dependence folding)</strong>。该方法使用一个保守依赖(conservative  dependence)表示循环分块携带的多个相邻依赖。通过该方法，为每个线程分配的同步变量个数不会超过嵌套循环的层数；此外，还给出了一个收益分析模型来选择最优的流水计算粒度。以下简称<strong>ENPA策略</strong>。</p>
</li>
<li>
<p>Liu在文章《面向共享存储结构的并行编译优化技术研究》的工作面向的也是规则DOACROSS循环中的单层并行性，设计了一个<strong>从循环层中寻找计算划分层和循环分块层的启发式算法</strong>，并且再次减少了每个线程的同步变量个数，限定为1。此外，她还提出了一个基于流水并行的代价模型，用以选择最佳流水计算粒度。以下简称<strong>CHPA策略</strong>。</p>
</li>
<li>
<p><strong>PLuTo</strong>是一款针对<strong>仿射嵌套循环的基于多面体模型的自动并行局部性优化</strong>工具。PLuTo通过为手工标记的<strong>C语言源代码</strong>寻找有效的仿射变换序列生成<strong>分块后的OpenMP</strong>并行代码，通过<strong>循环偏斜</strong>的方法对代码进行了转换，由转换后的并行程序在多核架构上可以获得较好的性能提升。以下简称PLuTo策略。</p>
</li>
</ul>
<p>在实际使用时，以上三种策略各有优劣，但均未能达到多核处理器并行加速效果的最佳值。</p>
<h3 id="现有技术"><a class="markdownIt-Anchor" href="#现有技术"></a> 现有技术</h3>
<h4 id="tls-简介"><a class="markdownIt-Anchor" href="#tls-简介"></a> TLS 简介</h4>
<blockquote>
<p>Steffan 等人所提出的<strong>猜测并行(Speculative Parallelization)<strong>技术为克服隐式编程模型的缺陷提供了解决的思路[15]。一般情况下，猜测并行技术指的是</strong>线程级猜测执行(Thread-Level Speculation, TLS)</strong>。TLS技术是一种<strong>运行时技术</strong>，允许程序员将多个并行段划分为多个计算任务，并在不考虑并行段之间相关的情况下，让这些计算任务并行执行。</p>
<p>具体来讲，不同于传统的依靠编译器来识别可以并行执行的串行代码段，TLS技术统一假设串行代码都可以由不同的线程并行执行。而为了保持串行程序的原本语义，系统会检测猜测线程是否已经使用了其前面的线程所产生的数据（检测数据相关冲突)，即根据顺序语义，执行前一部分程序代码的线程所更新的数据。数据相关冲突的检测和纠正主要通过硬件机制或软件机制或两者的组合来实现。如果检测到相关冲突，通常会丢弃使用了错误数据的线程所计算出的结果，并重新启动线程使用正确的数据进行计算。</p>
</blockquote>
<h4 id="基于-openmp-的自动并行化编译"><a class="markdownIt-Anchor" href="#基于-openmp-的自动并行化编译"></a> 基于 OpenMP 的自动并行化编译</h4>
<p>即将源码封装成有 OpenMP 编译制导语句的编译器</p>
<ol>
<li>iPat/OMP</li>
<li>Parallware</li>
<li><strong>PLUTO</strong></li>
<li>ROSE</li>
<li>S2P</li>
</ol>
<h6 id="openmp的优势"><a class="markdownIt-Anchor" href="#openmp的优势"></a> OpenMP的优势:</h6>
<ol>
<li>
<p>可移植的并行编译框架。</p>
</li>
<li>
<p>简单，不需要想 MPI并行编程框架一样处理消息传递。</p>
</li>
<li>
<p>数据构造和分解自动处理，不需要程序开发者关心。</p>
</li>
<li>
<p>在共享内存系统上可与MPI并行编程框架一起扩展。</p>
</li>
<li>
<p>与顺序执行兼容。加入 OpenMP编译指导的程序仍可以顺序执行，因为OpenMP将被编译器认为是程序中的注释。</p>
</li>
<li>
<p>粗粒度并行和细粒度并行都支持。</p>
</li>
<li>
<p>在众多加速器中可以使用。</p>
</li>
</ol>
<h6 id="openmp的缺点"><a class="markdownIt-Anchor" href="#openmp的缺点"></a> OpenMP的缺点:</h6>
<ol>
<li>引进 OpenMP后，程序同步故障和数据竞争条件将变得难以调试。</li>
<li>只能在共享内存的多处理器平台上应用。</li>
<li>需要编译器支持。</li>
<li>因只支持共享内存，扩展受到限制。</li>
<li>对compare-and-swap操作不支持117]。</li>
<li>没有可靠的错误处理机制。</li>
<li>没有控制线程处理映射的细粒度并行机制。</li>
<li>很有可能编写错误的共享的代码指示。</li>
<li>多线程的启动时间往往比单线程要长，因此如果程序足够短，则多线程执行将没有优势。</li>
</ol>
<h4 id="pluto"><a class="markdownIt-Anchor" href="#pluto"></a> PLUTO</h4>
<blockquote>
<p><strong>PLUTO</strong> is an <strong>automatic parallelization tool</strong> based on the <a target="_blank" rel="noopener" href="http://polyhedral.info/">polyhedral model</a>. The polyhedral model for compiler optimization provides an abstraction to perform high-level transformations such as <strong>loop-nest optimization</strong> and <strong>parallelization on affine loop nests</strong>. Pluto transforms C programs from <strong>source to source</strong> for <strong>coarse-grained parallelism</strong> and <strong>data locality simultaneously</strong>. The core transformation framework mainly works by <strong>finding affine transformations for efficient tiling</strong>. The scheduling algorithm used by Pluto has been published in [1]. <a target="_blank" rel="noopener" href="http://www.openmp.org/">OpenMP</a> parallel code for multicores can be automatically generated from sequential C program sections. Outer (communication-free), inner, or pipelined parallelization is achieved purely with OpenMP <em>parallel for</em> pragrams; the code is also <strong>optimized for locality</strong> and made amenable for <strong>auto-vectorization</strong>. An experimental evaluation and comparison with previous techniques can be found in [2]. Though the tool is fully automatic (C to OpenMP C), a number of options are provided (both command-line and through meta files) to tune aspects like <strong>tile sizes</strong>, <strong>unroll factors</strong>, and <strong>outer loop fusion structure</strong>. <a target="_blank" rel="noopener" href="http://www.cloog.org/">Cloog-ISL</a> is used for code generation.</p>
<p>To cite Pluto:</p>
<ol>
<li><a target="_blank" rel="noopener" href="http://www.csa.iisc.ernet.in/~uday/publications/uday-cc08.pdf">Automatic Transformations for Communication-Minimized Parallelization and Locality Optimization in the Polyhedral Model</a> [<a target="_blank" rel="noopener" href="http://www.csa.iisc.ernet.in/~uday/publications/uday-cc08.pdf">PDF</a> | <a target="_blank" rel="noopener" href="http://www.csa.iisc.ernet.in/~uday/uday.bib">BibTeX</a> ]<br />
Uday Bondhugula, M. Baskaran, S. Krishnamoorthy, J. Ramanujam, A. Rountev, and P. Sadayappan.<br />
<em>International Conference on Compiler Construction (ETAPS CC), Apr 2008, Budapest, Hungary.</em></li>
<li><a target="_blank" rel="noopener" href="http://dl.acm.org/authorize?N20736">A Practical Automatic Polyhedral Parallelizer and Locality Optimizer</a> [<a target="_blank" rel="noopener" href="http://dl.acm.org/authorize?N20736">PDF</a>| <a target="_blank" rel="noopener" href="http://www.csa.iisc.ernet.in/~uday/uday.bib">BibTeX</a>]<br />
Uday Bondhugula, A. Hartono, J. Ramanujan, P. Sadayappan.<br />
<em>ACM SIGPLAN Programming Languages Design and Implementation (PLDI), Jun 2008, Tucson, Arizona.</em></li>
</ol>
</blockquote>
<p>官网：<a target="_blank" rel="noopener" href="http://pluto-compiler.sourceforge.net/">http://pluto-compiler.sourceforge.net/</a></p>
<p>源码：<a target="_blank" rel="noopener" href="https://github.com/bondhugula/pluto">https://github.com/bondhugula/pluto</a></p>
<p>罕见较全面的中文评价：<a target="_blank" rel="noopener" href="https://www.zhihu.com/question/329294933">https://www.zhihu.com/question/329294933</a></p>
<p><img src="https://tuchuangrrr.oss-cn-shenzhen.aliyuncs.com/img/image-20201102205034698.png" srcset="/img/loading.gif" alt="image-20201102205034698" /></p>
<h4 id="ppcg"><a class="markdownIt-Anchor" href="#ppcg"></a> PPCG</h4>
<p><em>Polyhedral Parallel Code Generation for CUDA</em></p>
<blockquote>
<p>This article addresses the compilation of a sequential program for parallel execution on a modern GPU. To this end, we present a novel source-to-source compiler called <strong>PPCG</strong>. PPCG singles out for its ability to <strong>accelerate computations from any static control loop nest</strong>, <strong>generating multiple CUDA kernels when necessary</strong>. We introduce a <strong>multilevel tiling strategy</strong> and a <strong>code generation scheme for the parallelization</strong> and <strong>locality optimization of imperfectly nested loops</strong>, managing memory and exposing concurrency according to the constraints of modern GPUs. We evaluate our algorithms and tool on the entire PolyBench suite.</p>
</blockquote>
<p>源码：<a target="_blank" rel="noopener" href="https://github.com/Meinersbur/ppcg">https://github.com/Meinersbur/ppcg</a></p>
<h3 id="论文"><a class="markdownIt-Anchor" href="#论文"></a> 论文</h3>
<ul>
<li>Polyhedral Parallel Code Generation for CUDA
<ul>
<li><a target="_blank" rel="noopener" href="https://doi.org/10.1145/1375581.1375595">https://doi.org/10.1145/1375581.1375595</a></li>
<li>强相关，参考文献重要</li>
</ul>
</li>
<li>Non-affine Extensions to Polyhedral Code Generation
<ul>
<li><a target="_blank" rel="noopener" href="https://dl.acm.org/doi/10.1145/2581122.2544141">https://dl.acm.org/doi/10.1145/2581122.2544141</a></li>
</ul>
</li>
<li>A practical automatic polyhedral parallelizer and locality optimizer
<ul>
<li><a target="_blank" rel="noopener" href="https://doi.org/10.1145/1379022.1375595">https://doi.org/10.1145/1379022.1375595</a></li>
</ul>
</li>
<li>面向异构多核平台的高效能猜测
<ul>
<li>国防科大，王琼，博士论文，一堆并行的都是国防科大的</li>
<li>关键：面向 GPU 的线程猜测模型</li>
</ul>
</li>
<li>基于OpenMP的热点级猜测并行化编译研究
<ul>
<li>浙大，硕士论文</li>
<li>由介绍 OpenMP 架构、TLS 编译技术到实现热点级 TLS</li>
</ul>
</li>
<li>线程级猜测并行系统代码自动生成工具的设计与实现
<ul>
<li>国防科大，小篇幅</li>
<li>主要关于 HEUSPEC</li>
</ul>
</li>
</ul>
<h3 id="新闻"><a class="markdownIt-Anchor" href="#新闻"></a> 新闻</h3>
<h5 id="清华团队基于-polyhedral-模型获奖"><a class="markdownIt-Anchor" href="#清华团队基于-polyhedral-模型获奖"></a> 清华团队基于 <strong>Polyhedral</strong> 模型获奖</h5>
<p><strong>清华团队</strong>：<strong>Polyhedral</strong>模型来解决算子自动优化，通过研究循环分块和合并之间的更优组合，进一步利用芯片上的高速缓存，从而**提升程序在CPU、GPU以及AI芯片华为昇腾910等不同芯片上的性能。**链接：<a target="_blank" rel="noopener" href="https://zhidx.com/p/240883.html">https://zhidx.com/p/240883.html</a></p>
<blockquote>
<p>基于<strong>Polyhedral</strong>模型的编译技术，主要研究通过程序中循环嵌套的变换和优化来提升程序性能。这种模型以开发程序的并行性和局部性为目的，将源程序编译或转换为能够在目标芯片上性能更高的等价程序或可执行代码。</p>
<p>Polyhedral模型不需要手动编写调度，因此需在编译器中构建一个非常好的数学模型来计算调度。这种数学模型底层是通过解决整数线性规划问题来得到最优解的，但证书现象规划问题的求解过程是一个NP完全问题。</p>
<p>所以，<strong>如何构建Polyhedral模型的整数线性规划问题</strong>，使其能够达到手工编写调度性能相当的结果，是当前研究的核心难点。</p>
<p>另一难点在于如何<strong>降低该模型求解的复杂度</strong>。</p>
<p>Polyhedral模型涉及大量的基础数学理论，包括但不限于线性代数、空间几何、最优化理论和矩阵论，以及基础计算机理论科学的知识如图论、数理逻辑等。这些知识点本不复杂，但将它们统一地表示在一个模型里，导致该模型的理解难度大，入门的“门槛”比较高。</p>
</blockquote>
<h5 id="oneflow是如何做到世界最快深度学习框架的"><a class="markdownIt-Anchor" href="#oneflow是如何做到世界最快深度学习框架的"></a> OneFlow是如何做到世界最快深度学习框架的</h5>
<p>链接：<a target="_blank" rel="noopener" href="https://zhuanlan.zhihu.com/p/271740706">https://zhuanlan.zhihu.com/p/271740706</a></p>
<blockquote>
<p>近期，<a href="https://link.zhihu.com/?target=https%3A//github.com/Oneflow-Inc/oneflow/releases/tag/v0.2.0">OneFlow v0.2.0</a> 发布了，除了框架的易用性和完备性的提升在有序推进以外，最主要的更新就是多达17个性能优化，使得CNN和BERT的自动混合精度（AMP，Auto Mixed Precision）训练速度大幅提升，不仅远超其他各个主要框架的官方实现，同时也超过了NVIDIA深度优化过的版本，成为在主流旗舰显卡（V100 16G）上训练ResNet50-v1.5和BERT-base模型最快的框架)</p>
<ul>
<li>DLPerf评测——公平、公正、可复现的深度学习框架性能大PK</li>
<li>OneFlow追求极致性能的系统架构设计优势</li>
<li><strong>OneFlow在GPU上的CUDA性能优化揭秘</strong></li>
</ul>
</blockquote>
<h3 id="自主学习"><a class="markdownIt-Anchor" href="#自主学习"></a> 自主学习</h3>
<p>目前书籍：</p>
<ul>
<li>CUDA C 编程权威指南 (Professional CUDA C Programming)</li>
<li>并行算法设计及性能优化 (Parallel Computing and Performance Optimization)</li>
<li>串行算法并行化基础</li>
<li>支持推测并行化的多核事务存储体系结构研究</li>
<li>计算机体系结构量化研究方法 (Computer Architecture A Quantitative Approach)</li>
<li>结构化并行程序设计 高效计算模式 (Structured Parallel Programming)</li>
<li>并行编译方法 (国防科大 2000，国防科大的并行与分布式系列的书和论文，看起来基本是99年银河系列和15年天河系列超算的成果)</li>
<li>现代编译原理 (Modern Compiler Implementation in C)</li>
</ul>
<p>工具</p>
<ul>
<li><strong>ANTLR4</strong> 强大简单的词法语法分析器，极大得减轻了编译原理前端学习压力，不需要复杂的 yacc/lex</li>
<li>instaparse, parsec</li>
<li><strong>LLVM</strong>, LLVM 的 IR 设计得真好</li>
</ul>
<h3 id="困难"><a class="markdownIt-Anchor" href="#困难"></a> 困难</h3>
<ul>
<li>英文资料检索及阅读
<ul>
<li>略读关键词慢且不准确，无法快速筛选</li>
<li>解决：根据参考文献一连串挖</li>
</ul>
</li>
<li>大部分现有技术基于 CPU 架构
<ul>
<li>缺点：难以直接对应</li>
<li>优点：CPU 平台相对熟悉且简单，可以参照学习并行化相关技术</li>
</ul>
</li>
<li>各类 example 和 demo 与架构间有玄学相关性</li>
<li>编译原理有点麻烦（还好新时代好用的解析器变多了）</li>
</ul>

            </div>
            <hr>
            <div>
              <div class="post-metas mb-3">
                
                  <div class="post-meta mr-3">
                    <i class="iconfont icon-category"></i>
                    
                      <a class="hover-with-bg" href="/categories/%E6%9D%82/">杂</a>
                    
                  </div>
                
                
                  <div class="post-meta">
                    <i class="iconfont icon-tags"></i>
                    
                      <a class="hover-with-bg" href="/tags/%E5%B9%B6%E8%A1%8C/">并行</a>
                    
                  </div>
                
              </div>
              
              
                <div class="post-prevnext row">
                  <article class="post-prev col-6">
                    
                    
                      <a href="/2020/11/Buflab/">
                        <i class="iconfont icon-arrowleft"></i>
                        <span class="hidden-mobile">buflab</span>
                        <span class="visible-mobile">上一篇</span>
                      </a>
                    
                  </article>
                  <article class="post-next col-6">
                    
                    
                      <a href="/2020/11/bomblab/">
                        <span class="hidden-mobile">bomb lab</span>
                        <span class="visible-mobile">下一篇</span>
                        <i class="iconfont icon-arrowright"></i>
                      </a>
                    
                  </article>
                </div>
              
            </div>

            
              <!-- Comments -->
              <article class="comments" id="comments">
                
                
  <div id="vcomments"></div>
  <script type="text/javascript">
    function loadValine() {
      addScript('https://cdn.staticfile.org/valine/1.4.14/Valine.min.js', function () {
        new Valine({
          el: "#vcomments",
          app_id: "iAgkjaR2s2ws1XXtS5sVxE7H-MdYXbMMI",
          app_key: "vKka6XRlblVTkbKz8jLXx6EQ",
          placeholder: "你是来自评论区的小惊喜吗 ...",
          path: window.location.pathname,
          avatar: "retro",
          meta: ["nick","mail"],
          pageSize: "10",
          lang: "zh-CN",
          highlight: false,
          recordIP: true,
          serverURLs: "",
        });
      });
    }
    waitElementVisible('vcomments', loadValine);
  </script>
  <noscript>Please enable JavaScript to view the <a target="_blank" href="https://valine.js.org" rel="nofollow noopener noopener">comments
      powered by Valine.</a></noscript>


<!-- 
  <div id="vcomments"></div>
  <script type="text/javascript">
    function loadValine() {
      addScript('https://cdn.staticfile.org/valine/1.4.14/Valine.min.js', function () {
        new Valine({
          el: "#vcomments",
          enableQQ: true,
          boolean: true,
          app_id: "iAgkjaR2s2ws1XXtS5sVxE7H-MdYXbMMI",
          app_key: "vKka6XRlblVTkbKz8jLXx6EQ",
          placeholder: "你是来自评论区的小惊喜吗 ...",
          path: "window.location.pathname",
          avatar: "retro",
          meta: "["nick","mail"]",
          pageSize: "10",
          lang: "zh-CN",
          highlight: false,
          recordIP: true,
          serverURLs: "",
            // 设置Bilibili表情包地址
            emojiCDN: '//i0.hdslb.com/bfs/emote/', 
            // 表情title和图片映射
            emojiMaps: {
            "tv_doge": "6ea59c827c414b4a2955fe79e0f6fd3dcd515e24.png",
            "tv_亲亲": "a8111ad55953ef5e3be3327ef94eb4a39d535d06.png",
            "tv_偷笑": "bb690d4107620f1c15cff29509db529a73aee261.png",
            "tv_再见": "180129b8ea851044ce71caf55cc8ce44bd4a4fc8.png",
            "tv_冷漠": "b9cbc755c2b3ee43be07ca13de84e5b699a3f101.png",
            "tv_发怒": "34ba3cd204d5b05fec70ce08fa9fa0dd612409ff.png",
            "tv_发财": "34db290afd2963723c6eb3c4560667db7253a21a.png",
            "tv_可爱": "9e55fd9b500ac4b96613539f1ce2f9499e314ed9.png",
            "tv_吐血": "09dd16a7aa59b77baa1155d47484409624470c77.png",
            "tv_呆": "fe1179ebaa191569b0d31cecafe7a2cd1c951c9d.png",
            "tv_呕吐": "9f996894a39e282ccf5e66856af49483f81870f3.png",
            "tv_困": "241ee304e44c0af029adceb294399391e4737ef2.png",
            "tv_坏笑": "1f0b87f731a671079842116e0991c91c2c88645a.png",
            "tv_大佬": "093c1e2c490161aca397afc45573c877cdead616.png",
            "tv_大哭": "23269aeb35f99daee28dda129676f6e9ea87934f.png",
            "tv_委屈": "d04dba7b5465779e9755d2ab6f0a897b9b33bb77.png",
            "tv_害羞": "a37683fb5642fa3ddfc7f4e5525fd13e42a2bdb1.png",
            "tv_尴尬": "7cfa62dafc59798a3d3fb262d421eeeff166cfa4.png",
            "tv_微笑": "70dc5c7b56f93eb61bddba11e28fb1d18fddcd4c.png",
            "tv_思考": "90cf159733e558137ed20aa04d09964436f618a1.png",
            "tv_惊吓": "0d15c7e2ee58e935adc6a7193ee042388adc22af.png",
            "tv_打脸": "56ab10b624063e966bfcb76ea5dc4794d87dfd47.png",
            "tv_抓狂": "fe31c08edad661d63762b04e17b8d5ae3c71a757.png",
            "tv_抠鼻": "c666f55e88d471e51bbd9fab9bb308110824a6eb.png",
            "tv_斜眼笑": "911f987aa8bc1bee12d52aafe62bc41ef4474e6c.png",
            "tv_无奈": "ea8ed89ee9878f2fece2dda0ea8a5dbfe21b5751.png",
            "tv_晕": "5443c22b4d07fb1907ccc610c8e6db254f2461b7.png",
            "tv_流汗": "cead1c351ab8d79e9f369605beb90148db0fbed3.png",
            "tv_流泪": "7e71cde7858f0cd50d74b0264aa26db612a8a167.png",
            "tv_流鼻血": "c32d39db2737f89b904ca32700d140a9241b0767.png",
            "tv_点赞": "f85c354995bd99e28fc76c869bfe42ba6438eff4.png",
            "tv_生气": "26702dcafdab5e8225b43ffd23c94ac1ff932654.png",
            "tv_生病": "8b0ec90e6b86771092a498c54f09fc94621c1900.png",
            "tv_疑问": "0793d949b18d7be716078349c202c15ff166f314.png",
            "tv_白眼": "c1d59f439e379ee50eef488bcb5e5378e5044ea4.png",
            "tv_皱眉": "72ccad6679fea0d14cce648b4d818e09b8ffea2d.png",
            "tv_目瞪口呆": "0b8cb81a68de5d5365212c99375e7ace3e7891b7.png",
            "tv_睡着": "8b196675b53af58264f383c50ad0945048290b33.png",
            "tv_笑哭": "1abc628f6d4f4caf9d0e7800878f4697abbc8273.png",
            "tv_腼腆": "89712c0d4af73e67f89e35cbc518420380a7f6f4.png",
            "tv_色": "61822c7e9aae5da76475e7892534545336b23a6f.png",
            "tv_调侃": "4bc022533ef31544ca0d72c12c808cf4a1cce3e3.png",
            "tv_调皮": "b9c41de8e82dd7a8515ae5e3cb63e898bf245186.png",
            "tv_鄙视": "6e72339f346a692a495b123174b49e4e8e781303.png",
            "tv_闭嘴": "c9e990da7f6e93975e25fd8b70e2e290aa4086ef.png",
            "tv_难过": "87f46748d3f142ebc6586ff58860d0e2fc8263ba.png",
            "tv_馋": "fc7e829b845c43c623c8b490ee3602b7f0e76a31.png",
            "tv_鬼脸": "0ffbbddf8a94d124ca2f54b360bbc04feb6bbfea.png",
            "tv_黑人问号": "45821a01f51bc867da9edbaa2e070410819a95b2.png",
            "tv_鼓掌": "1d21793f96ef4e6f48b23e53e3b9e42da833a0f6.png"
                // ... 更多表情
            }
        });
      });
    }
    createObserver(loadValine, 'vcomments');
  </script>
      <style>
        #vcomments textarea {
            box-sizing: border-box;
            background: url("/img/v2-f3d5c26e7ac0da80ee72b6dc178d6aa1_b005.gif") 100% 100% no-repeat;
        }
      </style>
  <noscript>Please enable JavaScript to view the <a target="_blank" href="https://valine.js.org" rel="nofollow noopener noopener">comments
      powered by Valine.</a></noscript>
 -->

              </article>
            
          </article>
        </div>
      </div>
    </div>
    
      <div class="d-none d-lg-block col-lg-2 toc-container" id="toc-ctn">
        <div id="toc">
  <p class="toc-header"><i class="iconfont icon-list"></i>&nbsp;目录</p>
  <div id="tocbot"></div>
</div>

      </div>
    
  </div>
</div>

<!-- Custom -->

  <div class="col-lg-7 mx-auto nopadding-md">
    <div class="container custom post-content mx-auto">
      <img src="/img/371905140-ROBOT-WAVING-400px-unscreen(1).gif" srcset="/img/loading.gif" class="rounded mx-auto d-block mt-5" style="width:200px; height:200px;" >
    </div>
  </div>


    
  </main>

  
    <a id="scroll-top-button" href="#" role="button">
      <i class="iconfont icon-arrowup" aria-hidden="true"></i>
    </a>
  

  
    <div class="modal fade" id="modalSearch" tabindex="-1" role="dialog" aria-labelledby="ModalLabel"
     aria-hidden="true">
  <div class="modal-dialog modal-dialog-scrollable modal-lg" role="document">
    <div class="modal-content">
      <div class="modal-header text-center">
        <h4 class="modal-title w-100 font-weight-bold">搜索</h4>
        <button type="button" id="local-search-close" class="close" data-dismiss="modal" aria-label="Close">
          <span aria-hidden="true">&times;</span>
        </button>
      </div>
      <div class="modal-body mx-3">
        <div class="md-form mb-5">
          <input type="text" id="local-search-input" class="form-control validate">
          <label data-error="x" data-success="v"
                 for="local-search-input">关键词</label>
        </div>
        <div class="list-group" id="local-search-result"></div>
      </div>
    </div>
  </div>
</div>
  

  

  

  <footer class="mt-5">
  <div class="text-center py-3">
    <div>
      <a href="https://hexo.io" target="_blank" rel="nofollow noopener"><span>Hexo</span></a>
      <i class="iconfont icon-love"></i>
      <a href="https://github.com/fluid-dev/hexo-theme-fluid" target="_blank" rel="nofollow noopener">
        <span>Fluid</span></a>
        <br>
        <span>©2019 - 2020 By</span>
        <a href="https://github.com/rainrime" target="_blank" rel="nofollow noopener"><span>kunzh</span></a>
    </div>
    
  <div class="statistics">
    
    

    
      
        <!-- 不蒜子统计PV -->
        <span id="busuanzi_container_site_pv" style="display: none">
            总访问量 
            <span id="busuanzi_value_site_pv"></span>
             次
          </span>
      
      
        <!-- 不蒜子统计UV -->
        <span id="busuanzi_container_site_uv" style="display: none">
            总访客数 
            <span id="busuanzi_value_site_uv"></span>
             人
          </span>
      
    
  </div>


    

    
  </div>
</footer>


<!-- SCRIPTS -->
<script  src="https://cdn.staticfile.org/jquery/3.4.1/jquery.min.js" ></script>
<script  src="https://cdn.staticfile.org/twitter-bootstrap/4.4.1/js/bootstrap.min.js" ></script>
<script  src="/js/debouncer.js" ></script>
<script  src="/js/main.js" ></script>

<!-- Plugins -->


  
    <script  src="/js/lazyload.js" ></script>
  



  



  <script defer src="https://cdn.staticfile.org/clipboard.js/2.0.6/clipboard.min.js" ></script>
  <script  src="/js/clipboard-use.js" ></script>



  <script defer src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js" ></script>





  <script  src="https://cdn.staticfile.org/tocbot/4.11.1/tocbot.min.js" ></script>
  <script>
    $(document).ready(function () {
      var boardCtn = $('#board-ctn');
      var boardTop = boardCtn.offset().top;

      tocbot.init({
        tocSelector: '#tocbot',
        contentSelector: '#post-body',
        headingSelector: 'h1,h2,h3,h4,h5,h6',
        linkClass: 'tocbot-link',
        activeLinkClass: 'tocbot-active-link',
        listClass: 'tocbot-list',
        isCollapsedClass: 'tocbot-is-collapsed',
        collapsibleClass: 'tocbot-is-collapsible',
        collapseDepth: 6,
        scrollSmooth: true,
        headingsOffset: -boardTop
      });
      if ($('.toc-list-item').length > 0) {
        $('#toc').css('visibility', 'visible');
      }
    });
  </script>



  <script  src="https://cdn.staticfile.org/typed.js/2.0.11/typed.min.js" ></script>
  <script>
    var typed = new Typed('#subtitle', {
      strings: [
        '  ',
        "Automatic Parallelization Note 01&nbsp;",
      ],
      cursorChar: "_",
      typeSpeed: 70,
      loop: false,
    });
    typed.stop();
    $(document).ready(function () {
      $(".typed-cursor").addClass("h2");
      typed.start();
    });
  </script>



  <script  src="https://cdn.staticfile.org/anchor-js/4.2.2/anchor.min.js" ></script>
  <script>
    anchors.options = {
      placement: "right",
      visible: "hover",
      
      icon: "❡"
      
    };
    var el = "h1,h2,h3,h4,h5,h6".split(",");
    var res = [];
    for (item of el) {
      res.push(".markdown-body > " + item)
    }
    anchors.add(res.join(", "))
  </script>



  <script  src="/js/local-search.js" ></script>
  <script>
    var path = "/local-search.xml";
    var inputArea = document.querySelector("#local-search-input");
    inputArea.onclick = function () {
      searchFunc(path, 'local-search-input', 'local-search-result');
      this.onclick = null
    }
  </script>



  <script  src="https://cdn.staticfile.org/fancybox/3.5.7/jquery.fancybox.min.js" ></script>
  <link  rel="stylesheet" href="https://cdn.staticfile.org/fancybox/3.5.7/jquery.fancybox.min.css" />

  <script>
    $('#post img:not(.no-zoom img, img[no-zoom]), img[zoom]').each(
      function () {
        var element = document.createElement('a');
        $(element).attr('data-fancybox', 'images');
        $(element).attr('href', $(this).attr('src'));
        $(this).wrap(element);
      }
    );
  </script>





  

  
    <!-- KaTeX -->
    <link  rel="stylesheet" href="https://cdn.staticfile.org/KaTeX/0.11.1/katex.min.css" />
  



  
  
    <script type="text/javascript">
      //定义获取词语下标
      var a_idx = 0;
      jQuery(document).ready(function ($) {
        //点击body时触发事件
        $("body").click(function (e) {
          //需要显示的词语
          var a = new Array("富强", "民主", "文明", "和谐", "自由", "平等", "公正", "法治", "爱国", "敬业", "诚信", "友善");
          //设置词语给span标签
          var $i = $("<span/>").text(a[a_idx]);
          //下标等于原来下标+1  余 词语总数
          a_idx = (a_idx + 1) % a.length;
          //获取鼠标指针的位置，分别相对于文档的左和右边缘。
          //获取x和y的指针坐标
          var x = e.pageX, y = e.pageY;
          //在鼠标的指针的位置给$i定义的span标签添加css样式
          $i.css({
            "z-index": 999,
            "top": y - 20,
            "left": x,
            "position": "absolute",
            "font-weight": "bold",
            "color": rand_color()
          });
          // 随机颜色
          function rand_color() {
            return "rgb(" + ~~(255 * Math.random()) + "," + ~~(255 * Math.random()) + "," + ~~(255 * Math.random()) + ")"
          }
          //在body添加这个标签
          $("body").append($i);
          //animate() 方法执行 CSS 属性集的自定义动画。
          //该方法通过CSS样式将元素从一个状态改变为另一个状态。CSS属性值是逐渐改变的，这样就可以创建动画效果。
          //详情请看http://www.w3school.com.cn/jquery/effect_animate.asp
          $i.animate({
            //将原来的位置向上移动180
            "top": y - 180,
            "opacity": 0
            //1500动画的速度
          }, 1500, function () {
            //时间到了自动删除
            $i.remove();
          });
        });
      })
      ;
    </script>
  



  <script>(function (i, s, o, g, r, a, m) {
      i['DaoVoiceObject'] = r;
      i[r] = i[r] ||
        function () {
          (i[r].q = i[r].q || []).push(arguments);
        };
      i[r].l = 1 * new Date();
      a = s.createElement(o);
      m = s.getElementsByTagName(o)[0];
      a.async = 1;
      a.src = g;
      a.charset = 'utf-8';
      m.parentNode.insertBefore(a, m);
    })(window, document, 'script', ('https:' === document.location.protocol ? 'https:' : 'http:') + "//widget.daovoice.io/widget/244aa65c.js", 'daovoice');
    daovoice('init', {
      app_id: "244aa65c",
    });
    daovoice('update');
  </script>






  <script  src="https://cdn.staticfile.org/mermaid/8.5.0/mermaid.min.js" ></script>
  <script>
    if (window.mermaid) {
      mermaid.initialize({"theme":"default"});
    }
  </script>




  

  

  

  

  

  





</body>
</html>
